﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using LearnNet.BLL;
using LearnNet.Common;
using LearnNet.Entity;
using System.Text.RegularExpressions;
namespace LearnNet.Forum
{
    public  class CollectData
    {
        /// <summary>
        /// 采集连接地址
        /// </summary>
        public Entity.Collect_PageGetLinksInfo getLinksInfo = new Collect_PageGetLinksInfo();

        protected const string strSplit = "&&split&&";

        /// <summary>
        /// 采集具体内容
        /// </summary>
        public Entity.Collect_PageSetInfo pageInfo;


        protected   Regex regValue;
        protected   Regex reg;

        public void GetLinks()
        {
            string strUrl=string.Empty;
            
            Uri baseUri = new Uri(getLinksInfo.Url);
            string content;
            for (int i = getLinksInfo.StartNum; i < getLinksInfo.EndNum; i = i + getLinksInfo.AddNum)
            {
                strUrl = getLinksInfo.Url.Replace("{*}", i.ToString());
                content = Utils.GetRequestString(strUrl, 8000, 1, getLinksInfo.PageEncoding);
                GetLinkHref(content, getLinksInfo.IncludeStr, string.Concat(baseUri.Scheme, "://", baseUri.DnsSafeHost));
            }
           
        }

        /// <summary>
        /// 匹配所有绝对超链接地址
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="str_has">超链接中包括</param>
        /// <param name="oldUrl">获取地址</param>
        /// <returns></returns>
        public void GetLinkHref(string inputstring, string str_has, string baseUrl)
        {
            Common.RegExpression.LinkRegExpression linkRg = new LearnNet.Common.RegExpression.LinkRegExpression();
            Common.RegExpression.RegContext regContent = new LearnNet.Common.RegExpression.RegContext(inputstring, 'M', "");
            linkRg.Evaluate(regContent);
            Common.RegExpression.IRegExpression reg;
            Common.RegExpression.RegContext regTemp;
            string a_url = string.Empty;
           
            foreach (string strmatch in regContent.Matches)
            {
                reg = new LearnNet.Common.RegExpression.HrefRegExpression();
                regTemp = new LearnNet.Common.RegExpression.RegContext(strmatch, 'M', "");
             

                if (reg.IsMatch(strmatch))
                {
                    reg.Evaluate(regTemp);
                    a_url =regTemp.ToString().Replace("href=", "").Replace("HREF=", "").Replace("'", "").Replace("\"", "");
                }
                if (a_url.IndexOf("http://") == -1)
                {
                    if (a_url.StartsWith("/"))
                    {
                        a_url = string.Concat(baseUrl, a_url);
                    }
                    else
                    {
                        a_url = string.Concat(baseUrl,"/", a_url);
                    }
                }
                regValue = new Regex("#[^# ]*[ ]*", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_url))
                {
                    a_url = a_url.Replace(regValue.Match(a_url).Value, "");
                }
                regValue = new Regex(str_has, RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_url))
                {
                    getLinksInfo.AddPage(false, a_url);

                }
            }

        }

        /// <summary>
        /// 获取类似第一个meta的标签
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="tagName">获取标签名</param>
        /// <param name="attrname">标签类型如:name</param>
        /// <param name="key">标签类型值如:keywords</param>
        /// <returns>字符串</returns>
        public string GetSingleTagValue(string inputstring, string tagName, string attrname, string key)
        {
            Regex reg = new Regex("<[ ]*" + tagName + " [^<>]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string result = string.Empty;
            foreach (Match match in matchs)
            {
                string matchValue = match.Value;
                regValue = new Regex("content=[\"|'][^'\"]*[\"|']", RegexOptions.IgnoreCase);
                if ((matchValue.ToLower().IndexOf(attrname.ToLower() + "=\"" + key.ToLower() + "\"") != -1) || (matchValue.ToLower().IndexOf(attrname.ToLower() + "='" + key.ToLower() + "'") != -1))
                {
                    if (regValue.IsMatch(matchValue))
                    {
                        result = regValue.Match(matchValue).Value;
                        if (!string.IsNullOrEmpty(result))
                        {
                            result = result.Replace("CONTENT=", "").Replace("content=", "").Replace("\"", "").Replace("'", "").Trim();
                        }
                    }
                    return result;
                }
            }
            return string.Empty;
        }

         /// <summary>
        /// 获取类似title标签的值
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="tagName">获取标签名</param>
        /// <returns>返回字符串</returns>
        public string GetFixTagContent(string inputstring, string tagName)
        {
            reg = new Regex("<[ ]*" + tagName + ".+?" + tagName + "[ ]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string result = string.Empty;
            string a_content = string.Empty;
            string a_content_left = string.Empty;
            string a_content_right = string.Empty;
            foreach (Match match in matchs)
            {
                a_content = match.Value;

                regValue = new Regex("<[ ]*" + tagName + "[^<>]*>", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_content))
                {
                    a_content_left = regValue.Match(a_content).Value;
                }
                regValue = new Regex("</[ ]*" + tagName + "[ ]*>", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_content))
                {
                    a_content_right = regValue.Match(a_content).Value;
                }
                if (a_content.Length > 0 && a_content_left != "" && a_content_right != "")
                {
                    result = a_content.Replace(a_content_left, "").Replace(a_content_right, "");
                }
                return result;
            }
            return result;
        }
        /// <summary>
        /// 返回超链接文本
        /// </summary>
        /// <param name="inputstring">含有连接的字符窜</param>
        /// <param name="strInclude">超链接中包含</param>
        /// <returns>string</returns>
        public List<string> GetLinkText(string inputstring, string strInclude, Int32 maxListLength)
        {
            reg = new Regex("<[ ]*a.+?a[ ]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string a_content = string.Empty;
          
            List<string> lst = new List<string>();
            foreach (Match match in matchs)
            {
                a_content = match.Value;

                if (a_content.ToLower().IndexOf(strInclude.ToLower()) != -1)
                {
                    // HttpContext.Current.Response.Write(str_has+a_content+"<br/>__________________________");

                    regValue = new Regex("<[ ]*a[^<>]>", RegexOptions.IgnoreCase);
                    if (regValue.IsMatch(a_content))
                    {
                        a_content = a_content.Replace(regValue.Match(a_content).Value, "");
                    }
                    regValue = new Regex("<[ ]*a[ ]*/[ ]*>", RegexOptions.IgnoreCase);
                    if (regValue.IsMatch(a_content))
                    {
                        a_content = a_content.Replace(regValue.Match(a_content).Value, "");
                    }
                    a_content =Utils.RemoveHtml(a_content);
                    if (!lst.Contains(a_content))
                    {
                        if (maxListLength == 0)
                        {
                            lst.Add(a_content);
                        }
                        else
                        {
                            if (lst.Count <= maxListLength)
                            {
                                lst.Add(a_content);
                            }
                            else
                            {
                                break;
                            }
                        }
                    }
                }
            }

            return lst;
        }

        /// <summary>
        /// 获取指定字符串间的字符串
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="leftTag">开始标记</param>
        /// <param name="rightTag">结束标记</param>
        /// <param name="cutStartNum">开始阶段去除的字符数</param>
        /// <param name="cutStrNum">最后阶段去除的字符数</param>
        /// <param name="isNoHtml">是否包含html标记</param>
        /// <param name="isCut">是否去除开始和结束标记，和最后移除字符数</param>
        /// <returns>返回字符串</returns>
        public string GetCutString(string inputstring, string leftTag, string rightTag, int cutStartNum, int cutStrNum, bool isNoHtml, bool isCut)
        {
            reg = new Regex(leftTag + ".*?" + rightTag, RegexOptions.IgnoreCase);
            string str = reg.Match(inputstring).Value;
            if (str.Length > cutStrNum)
            {
                if (isCut)
                {
                    str = str.Replace(leftTag, "").Replace(rightTag, "");
                    str = str.Substring(0, (str.Length - cutStrNum));
                    str = str.Substring(cutStartNum, (str.Length - cutStartNum));
                }
                if (isNoHtml)
                {
                    str =Utils.RemoveHtml(str);
                }
            }
            return str;

        }

        /// <summary>
        /// 移除符合条件的字符串
        /// </summary>
        /// <param name="inputstring">要检测的字符串</param>
        /// <param name="RemoveRegex">移除的正则表达式</param>
        /// <returns>string</returns>
        public string RemoveTag(string inputstring, string RemoveRegex)
        {
            reg = new Regex(RemoveRegex, RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            foreach (Match match in matchs)
            {
                inputstring = inputstring.Replace(match.Value, string.Empty);
            }
            return inputstring;
        }

        /// <summary>
        /// 获取采集内容
        /// </summary>
        /// <param name="url"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public void GetContent(string url, System.Text.Encoding encoding)
        {
            string inputstring = Utils.GetRequestString(url, 8000, 0, encoding);
            pageInfo.Title = GetFixTagContent(inputstring, "title").Replace(pageInfo.TitleReplaceStr, pageInfo.TitleReplasedStr);
            //根据标题获取百度关键词
            if (pageInfo.IsAutoKeyWord)
            {

                string PageUrl = "http://www.google.com.hk/search?q=" + pageInfo.Title;
                string getHtml =Utils.GetRequestString(PageUrl, 800, 0, System.Text.Encoding.GetEncoding("gb2312"));
                List<string> lstgoogle = GetLinkText(getHtml, "s?wd", 3);
                string kewordTemp = Utils.GetRequestString("http://www.baidu.com/s?lm=0&si=&tn=yang0000meng_pg&rn=10&ie=utf-8&ct=0&wd=" + pageInfo.Title + "&pn=10&ver=0&cl=3&uim=1&usm=0", 8000, 0, System.Text.Encoding.GetEncoding("gb2312"));
                List<string> lstbaidu=GetLinkText(kewordTemp, "s?wd",1);
                List<string> keys=Utils.CombineList(lstgoogle,lstbaidu);
                pageInfo.KeyWord = Utils.ListToString(keys, ",", 4).Replace("[1]", string.Empty).Replace("[2]", string.Empty).Replace("[3]", string.Empty).Replace("[4]", string.Empty).Replace("[5]", string.Empty).Replace("[6]", string.Empty).Replace("[7]", string.Empty).Replace("[8]", string.Empty).Replace("[9]", string.Empty).Replace("[10]", string.Empty).Replace("上一页", string.Empty).Replace("[11]", string.Empty).Replace("下一页", string.Empty);
                
            }
            if (pageInfo.IsAutoDescription)
            {
                pageInfo.Description = GetSingleTagValue(inputstring, "meta", "name", "description");
                pageInfo.Description = Utils.RemoveHtml(pageInfo.Description);
                pageInfo.Description = Utils.ReplaceArray(pageInfo.Description, pageInfo.DescriptionReplaseStr, pageInfo.DescriptionReplasedStr, strSplit, false);
            }
            pageInfo.Content = GetCutString(inputstring, pageInfo.ContentMatchLeft, pageInfo.ContentMatchRight, pageInfo.ContentRemoveLeftLength, pageInfo.ContentRemoveRightLength, pageInfo.IsIncludeHtml, true);
            if (pageInfo.IsIncludeHtml)
            {
                if (!pageInfo.IsIncludeClass)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, "<[ ]*a[ ]*[^>]*?>");
                    pageInfo.Content = RemoveTag(pageInfo.Content, "<[ ]*/[ ]*a[ ]*>");
                }
                if (!pageInfo.IsIncludeColor)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*:[^;'\"]+;");
                    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*=['|\"]+[^'\"]+");
                }
                //if (!pageInfo.IsIncludeFile)
                //{
                //    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*:[^;'\"]+;");
                //    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*:[^;'\"]+;");
                //}
                //if (!pageInfo.IsIncludeFrame)
                //{
                //    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*:[^;'\"]+;");
                //    pageInfo.Content = RemoveTag(pageInfo.Content, "color[ ]*:[^;'\"]+;");
                //}
                if (!pageInfo.IsIncludeImg)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, "<[ ]*img[^>]*?>");
                }
                if (!pageInfo.IsIncludeLink)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, "<[ ]*a[ ]*[^>]*?>");
                    pageInfo.Content = RemoveTag(pageInfo.Content, "<[ ]*/[ ]*a[ ]*>");
                }
                if (!pageInfo.IsIncludeScript)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, @"<[ ]*script[^>]*?>.*?<[ ]*/[ ]*script[ ]*>");
                }
                if (!pageInfo.IsIncludeStyle)
                {
                    pageInfo.Content = RemoveTag(pageInfo.Content, "style[ ]*=['|\"]+[^'|\"]*?['|\"]+");
                }

            }
            pageInfo.Content = Utils.ReplaceArrayToEmpty(pageInfo.Content, "<[ ]*div[ ]*[^>]*?>", "<p>", false);
            pageInfo.Content = Utils.ReplaceArrayToEmpty(pageInfo.Content, "<[ ]*/[ ]*div[ ]*>", "</p>", false);

        }




    }
}
